{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建计算图——LSTM模型\n",
    "#    embedding\n",
    "#    LSTM\n",
    "#    fc\n",
    "#    train_op\n",
    "# 训练流程代码\n",
    "# 数据集封装\n",
    "#    api: next_batch(batch_size)\n",
    "# 词表封装:\n",
    "#    api: sentence2id(text_sentence): 句子转换id\n",
    "# 类别的封装：\n",
    "#    api: category2id(text_category).\n",
    "\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_default_params():\n",
    "    return tf.contrib.training.HParams(\n",
    "        num_embedding_size = 16,\n",
    "        num_timesteps = 50,\n",
    "        num_filters = 64,\n",
    "        num_kernel_size = 3,\n",
    "        num_fc_nodes = 32,\n",
    "        batch_size = 100,\n",
    "        learning_rate = 0.001,\n",
    "        num_word_threshold = 10,\n",
    "    )\n",
    "\n",
    "hps = get_default_params()\n",
    "\n",
    "train_file = '../../text_classification_data/cnews.train.seg.txt'\n",
    "val_file = '../../text_classification_data/cnews.val.seg.txt'\n",
    "test_file = '../../text_classification_data/cnews.test.seg.txt'\n",
    "vocab_file = '../../text_classification_data/cnews.vocab.txt'\n",
    "category_file = '../../text_classification_data/cnews.category.txt'\n",
    "output_folder = './run_text_cnn'\n",
    "\n",
    "if not os.path.exists(output_folder):\n",
    "    os.mkdir(output_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocab:\n",
    "    def __init__(self, filename, num_word_threshold):\n",
    "        self._word_to_id = {}\n",
    "        self._unk = -1\n",
    "        self._num_word_threshold = num_word_threshold\n",
    "        self._read_dict(filename)\n",
    "    \n",
    "    def _read_dict(self, filename):\n",
    "        with open(filename, 'r') as f:\n",
    "            lines = f.readlines()\n",
    "        for line in lines:\n",
    "            word, frequency = line.strip('\\r\\n').split('\\t')\n",
    "            word = word.decode('utf-8')\n",
    "            frequency = int(frequency)\n",
    "            if frequency < self._num_word_threshold:\n",
    "                continue\n",
    "            idx = len(self._word_to_id)\n",
    "            if word == '<UNK>':\n",
    "                self._unk = idx\n",
    "            self._word_to_id[word] = idx\n",
    "    \n",
    "    def word_to_id(self, word):\n",
    "        return self._word_to_id.get(word, self._unk)\n",
    "    \n",
    "    @property\n",
    "    def unk(self):\n",
    "        return self._unk\n",
    "    \n",
    "    def size(self):\n",
    "        return len(self._word_to_id)\n",
    "    \n",
    "    def sentence_to_id(self, sentence):\n",
    "        word_ids = [self.word_to_id(cur_word) \\\n",
    "                    for cur_word in sentence.split()]\n",
    "        return word_ids\n",
    "\n",
    "\n",
    "class CategoryDict:\n",
    "    def __init__(self, filename):\n",
    "        self._category_to_id = {}\n",
    "        with open(filename, 'r') as f:\n",
    "            lines = f.readlines()\n",
    "        for line in lines:\n",
    "            category = line.strip('\\r\\n').decode('utf-8')\n",
    "            idx = len(self._category_to_id)\n",
    "            self._category_to_id[category] = idx\n",
    "    \n",
    "    def size(self):\n",
    "        return len(self._category_to_id)\n",
    "        \n",
    "    def category_to_id(self, category):\n",
    "        if not category in self._category_to_id:\n",
    "            raise Execption(\n",
    "                \"%s is not in our category list\" % category_name)\n",
    "        return self._category_to_id[category]\n",
    "        \n",
    "vocab = Vocab(vocab_file, hps.num_word_threshold)\n",
    "vocab_size = vocab.size()\n",
    "tf.logging.info('vocab_size: %d' % vocab_size)\n",
    "\n",
    "category_vocab = CategoryDict(category_file)\n",
    "num_classes = category_vocab.size()\n",
    "tf.logging.info('num_classes: %d' % num_classes)\n",
    "test_str = '时尚'\n",
    "tf.logging.info(\n",
    "    'label: %s, id: %d' % (\n",
    "        test_str,\n",
    "        category_vocab.category_to_id(test_str.decode('utf-8'))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextDataSet:\n",
    "    def __init__(self, filename, vocab, category_vocab, num_timesteps):\n",
    "        self._vocab = vocab\n",
    "        self._category_vocab = category_vocab\n",
    "        self._num_timesteps = num_timesteps\n",
    "        # matrix\n",
    "        self._inputs = []\n",
    "        # vector\n",
    "        self._outputs = []\n",
    "        self._indicator = 0\n",
    "        self._parse_file(filename)\n",
    "    \n",
    "    def _parse_file(self, filename):\n",
    "        tf.logging.info('Loading data from %s', filename)\n",
    "        with open(filename, 'r') as f:\n",
    "            lines = f.readlines()\n",
    "        for line in lines:\n",
    "            label, content = line.strip('\\r\\n').decode('utf-8').split('\\t')\n",
    "            id_label = self._category_vocab.category_to_id(label)\n",
    "            id_words = self._vocab.sentence_to_id(content)\n",
    "            id_words = id_words[0: self._num_timesteps]\n",
    "            padding_num = self._num_timesteps - len(id_words)\n",
    "            id_words = id_words + [\n",
    "                self._vocab.unk for i in range(padding_num)]\n",
    "            self._inputs.append(id_words)\n",
    "            self._outputs.append(id_label)\n",
    "        self._inputs = np.asarray(self._inputs, dtype = np.int32)\n",
    "        self._outputs = np.asarray(self._outputs, dtype = np.int32)\n",
    "        self._random_shuffle()\n",
    "    \n",
    "    def _random_shuffle(self):\n",
    "        p = np.random.permutation(len(self._inputs))\n",
    "        self._inputs = self._inputs[p]\n",
    "        self._outputs = self._outputs[p]\n",
    "    \n",
    "    def next_batch(self, batch_size):\n",
    "        end_indicator = self._indicator + batch_size\n",
    "        if end_indicator > len(self._inputs):\n",
    "            self._random_shuffle()\n",
    "            self._indicator = 0\n",
    "            end_indicator = batch_size\n",
    "        if end_indicator > len(self._inputs):\n",
    "            raise Execption(\"batch_size: %d is too large\" % batch_size)\n",
    "        \n",
    "        batch_inputs = self._inputs[self._indicator: end_indicator]\n",
    "        batch_outputs = self._outputs[self._indicator: end_indicator]\n",
    "        self._indicator = end_indicator\n",
    "        return batch_inputs, batch_outputs\n",
    "            \n",
    "train_dataset = TextDataSet(\n",
    "    train_file, vocab, category_vocab, hps.num_timesteps) \n",
    "val_dataset = TextDataSet(\n",
    "    val_file, vocab, category_vocab, hps.num_timesteps)\n",
    "test_dataset = TextDataSet(\n",
    "    test_file, vocab, category_vocab, hps.num_timesteps)\n",
    "\n",
    "print train_dataset.next_batch(2)\n",
    "print val_dataset.next_batch(2)\n",
    "print test_dataset.next_batch(2)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(hps, vocab_size, num_classes):\n",
    "    num_timesteps = hps.num_timesteps\n",
    "    batch_size = hps.batch_size\n",
    "    \n",
    "    inputs = tf.placeholder(tf.int32, (batch_size, num_timesteps))\n",
    "    outputs = tf.placeholder(tf.int32, (batch_size, ))\n",
    "    keep_prob = tf.placeholder(tf.float32, name = 'keep_prob')\n",
    "    \n",
    "    global_step = tf.Variable(\n",
    "        tf.zeros([], tf.int64), name = 'global_step', trainable=False)\n",
    "    \n",
    "    embedding_initializer = tf.random_uniform_initializer(-1.0, 1.0)\n",
    "    with tf.variable_scope(\n",
    "        'embedding', initializer = embedding_initializer):\n",
    "        embeddings = tf.get_variable(\n",
    "            'embedding',\n",
    "            [vocab_size, hps.num_embedding_size],\n",
    "            tf.float32)\n",
    "        # [1, 10, 7] -> [embeddings[1], embeddings[10], embeddings[7]]\n",
    "        embed_inputs = tf.nn.embedding_lookup(embeddings, inputs)\n",
    "    \n",
    "    scale = 1.0 / math.sqrt(hps.num_embedding_size + hps.num_filters) / 3.0\n",
    "    cnn_init = tf.random_uniform_initializer(-scale, scale)\n",
    "    with tf.variable_scope('cnn', initializer = cnn_init):\n",
    "        # embed_inputs: [batch_size, timesteps, embed_size]\n",
    "        # conv1d: [batch_size, timesteps, num_filters]\n",
    "        conv1d = tf.layers.conv1d(\n",
    "            embed_inputs,\n",
    "            hps.num_filters,\n",
    "            hps.num_kernel_size,\n",
    "            activation = tf.nn.relu,\n",
    "        )\n",
    "        global_maxpooling = tf.reduce_max(conv1d, axis=[1])\n",
    "        \n",
    "    \"\"\"\n",
    "    scale = 1.0 / math.sqrt(hps.num_embedding_size + hps.num_lstm_nodes[-1]) / 3.0\n",
    "    lstm_init = tf.random_uniform_initializer(-scale, scale)\n",
    "    with tf.variable_scope('lstm_nn', initializer = lstm_init):\n",
    "        cells = []\n",
    "        for i in range(hps.num_lstm_layers):\n",
    "            cell = tf.contrib.rnn.BasicLSTMCell(\n",
    "                hps.num_lstm_nodes[i],\n",
    "                state_is_tuple = True)\n",
    "            cell = tf.contrib.rnn.DropoutWrapper(\n",
    "                cell,\n",
    "                output_keep_prob = keep_prob)\n",
    "            cells.append(cell)\n",
    "        cell = tf.contrib.rnn.MultiRNNCell(cells)\n",
    "        \n",
    "        initial_state = cell.zero_state(batch_size, tf.float32)\n",
    "        # rnn_outputs: [batch_size, num_timesteps, lstm_outputs[-1]]\n",
    "        rnn_outputs, _ = tf.nn.dynamic_rnn(\n",
    "            cell, embed_inputs, initial_state = initial_state)\n",
    "        last = rnn_outputs[:, -1, :]\n",
    "    \"\"\"\n",
    "    \n",
    "    fc_init = tf.uniform_unit_scaling_initializer(factor=1.0)\n",
    "    with tf.variable_scope('fc', initializer = fc_init):\n",
    "        fc1 = tf.layers.dense(global_maxpooling, \n",
    "                              hps.num_fc_nodes,\n",
    "                              activation = tf.nn.relu,\n",
    "                              name = 'fc1')\n",
    "        fc1_dropout = tf.contrib.layers.dropout(fc1, keep_prob)\n",
    "        logits = tf.layers.dense(fc1_dropout,\n",
    "                                 num_classes,\n",
    "                                 name = 'fc2')\n",
    "    \n",
    "    with tf.name_scope('metrics'):\n",
    "        softmax_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "            logits = logits, labels = outputs)\n",
    "        loss = tf.reduce_mean(softmax_loss)\n",
    "        # [0, 1, 5, 4, 2] -> argmax: 2\n",
    "        y_pred = tf.argmax(tf.nn.softmax(logits),\n",
    "                           1, \n",
    "                           output_type = tf.int32)\n",
    "        correct_pred = tf.equal(outputs, y_pred)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "    \n",
    "    with tf.name_scope('train_op'):\n",
    "        train_op = tf.train.AdamOptimizer(hps.learning_rate).minimize(\n",
    "            loss, global_step=global_step)\n",
    "        \"\"\"\n",
    "        tvars = tf.trainable_variables()\n",
    "        for var in tvars:\n",
    "            tf.logging.info('variable name: %s' % (var.name))\n",
    "        grads, _ = tf.clip_by_global_norm(\n",
    "            tf.gradients(loss, tvars), hps.clip_lstm_grads)\n",
    "        optimizer = tf.train.AdamOptimizer(hps.learning_rate)\n",
    "        train_op = optimizer.apply_gradients(\n",
    "            zip(grads, tvars), global_step = global_step)\n",
    "        \"\"\"\n",
    "    \n",
    "    return ((inputs, outputs, keep_prob),\n",
    "            (loss, accuracy),\n",
    "            (train_op, global_step))\n",
    "\n",
    "placeholders, metrics, others = create_model(\n",
    "    hps, vocab_size, num_classes)\n",
    "\n",
    "inputs, outputs, keep_prob = placeholders\n",
    "loss, accuracy = metrics\n",
    "train_op, global_step = others\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_op = tf.global_variables_initializer()\n",
    "train_keep_prob_value = 0.8\n",
    "test_keep_prob_value = 1.0\n",
    "\n",
    "num_train_steps = 10000\n",
    "\n",
    "# Train: 100%\n",
    "# Valid: 95.7%\n",
    "# Test:  95.3%\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init_op)\n",
    "    for i in range(num_train_steps):\n",
    "        batch_inputs, batch_labels = train_dataset.next_batch(\n",
    "            hps.batch_size)\n",
    "        outputs_val = sess.run([loss, accuracy, train_op, global_step],\n",
    "                               feed_dict = {\n",
    "                                   inputs: batch_inputs,\n",
    "                                   outputs: batch_labels,\n",
    "                                   keep_prob: train_keep_prob_value,\n",
    "                               })\n",
    "        loss_val, accuracy_val, _, global_step_val = outputs_val\n",
    "        if global_step_val % 20 == 0:\n",
    "            tf.logging.info(\"Step: %5d, loss: %3.3f, accuracy: %3.3f\"\n",
    "                            % (global_step_val, loss_val, accuracy_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
